import scrapy
from scrapy import Request
from scrapyspider.items import Urls

class urlsSpider(scrapy.Spider):
    name = "urls"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36',
    }
    def start_requests(self):
        url = 'https://www.saikr.com/vs'
        yield Request(url, headers=self.headers)

    def parse(self, response):
        item = Urls()
        selectors = response.xpath('//h3[@class="tit"]')
        for selector in selectors:
            url = selector.xpath('./a/@href').extract()
            item['url'] = "".join(url)
            yield item

        next_url = response.xpath('//li[@class="next"]/a/@href').extract()
        if (next_url):
            next_url = 'https://www.saikr.com' + next_url[0].strip()
            yield Request(next_url, headers=self.headers)